﻿"use strict"

/**
 * 调度模块，主要获取所有的爬虫定义，然后分别调用爬虫，最后将数据保存，并发送邮件
 * sunny
 *  
 */
import * as fs from "fs";
import * as path from "path";
import * as child_process from "child_process";
import { Base, Item } from "../modules/base";
import { Driver } from "../db";
import { setTimeout } from "timers";
import { on } from "cluster";
let deasync = require('deasync');

const spiderStateCollection = "spider_states";
const dbinfoCollection = "dbinfo";

let schdlObject: any = {
    name: "crawler scheduler",
    option: "all",
    testmodule: null //测试模块
};

let modules_files = fs.readdirSync(__dirname + "/../modules");
let modules: any[] = [];

// 载入所有有效的爬虫模块
let load_modules = (): void => {
    let module_path = "../modules/";
    if (schdlObject.category) {
        modules_files = fs.readdirSync(__dirname + "/../modules/" + schdlObject.category);
        module_path = module_path + schdlObject.category + "/";
    } else {
        modules_files = fs.readdirSync(__dirname + "/../modules");
    }
    for (var i = 0, j = modules_files.length; i < j; i++) {
        var fn = modules_files[i];
        if (/.js$/.test(fn)) {
            var module_name = fn.substring(0, fn.length - 3);
            if (module_name == "base") continue;
            var m = require(module_path + module_name);
            m.module_name = module_name;
            if (m) {
                console.log((m["name"] || "") + " [" + module_name + "] loaded.");
                if (m.hasOwnProperty('enabled')) {
                    if (m.enabled == true)
                        modules.push(m);
                } else {
                    modules.push(m);
                }
            }
        }
    }
};

// 判断是否需要保存
let recordHanlder = (module: Base, item: Item): void => {
    let keyField = module.keyField;
    var keyValue = (item as any)[keyField];
    var driver = new Driver();
    let rs = driver.findSync((<any>schdlObject).collection || item["_collection"] || "others", { keyField: keyValue });
    if (rs.length > 0) {
        console.log("item " + keyValue + " exists");
    } else {
        driver.insertSync((<any>schdlObject).collection || item["_collection"] || "others", [item]);
        console.log("item " + keyValue + " inserted");
    }
}

// 每个条目的处理
let itemsHandle = (module: Base, items: Item[], state: SpiderState): void => {
    let start = state.currentItemIndex || 0;
    let total = items.length;
    for (let i = start, j = total; i < j; i++) {
        let item = items[i];
        if (schdlObject.testmodule) {
            //do nothing;
        } else {
            recordHanlder(module, item);
            state.currentItemIndex = i + 1;
            stateStore(state);
        }
    }
};

// 爬虫状态
class SpiderState {
    title: string;
    page: number;
    currentPage: number;
    currentItemIndex: number;
    constructor(name: string) {
        this.page = 0;
        this.currentPage = 0;
        this.title = name;
    }
}


// 保存爬虫状态
let stateStore = (state: SpiderState): SpiderState => {
    let driver = new Driver();
    try {
        let eitems = driver.findSync(spiderStateCollection, { title: state.title });
        if (eitems && eitems.length > 0) {
            driver.updateSync(spiderStateCollection, { title: state.title }, state);
        } else {
            driver.insertSync(spiderStateCollection, [state]);
        }
    } catch (e) {
        console.error(e);
    }
    return state;
};

// 载入爬虫状态
let stateLoad = (spidername: string): SpiderState => {
    let driver = new Driver();
    let baseObject: SpiderState;
    try {
        //因为爬虫名即为数据库中爬虫表名，所以直接传入爬虫名称查询表中数据是否存在，如果length大于0，则为已经爬过的网站
        let states = driver.findSync(spiderStateCollection, { title: spidername });
        if (states && states.length > 0) {
            baseObject = states[0];
        } else {
            baseObject = new SpiderState(spidername);
        }
    } catch (e) {
        console.error(e.message);
        baseObject = new SpiderState(spidername);
    }
    return baseObject;
};

// 每个爬虫的处理
let moduleHandle = function (m: any, mode: string): void {
    console.log(m.name + " " + m.version);
    if (mode == "test") {
        let page = m.getPageCount(); // getPageCount
        console.log('total page', page);
        let start = 0;
        for (let i = start, j = page; i < j; i++) {
            let items = m.getItems(i);
            console.log('page: ', (i + 1), ' items count: ', items.length);
        }
    } else {
        let state = stateLoad(m.name);
        let page = m.getPageCount(); // getPageCount
        state.page = page;
        let start = state.currentPage;
        if (start != 0) {
            console.log("spider resume, start page with: " + start + " item index from: " + state.currentItemIndex);
        }
        if (start > 0) { // 如果从新页面开始，为了安全，还是先获取前两页，以便构建好session等信息
            m.getItems(0);
            if (page > 1) {
                m.getItems(1);
            }
        }
        for (let i = start, j = page; i < j; i++) {
            let items = m.getItems(i);
            itemsHandle(m, items, state);
            state.currentItemIndex = 0;
            state.currentPage = i + 1;
            stateStore(state);
        }
    }
};
//爬虫总表名
//collection_name 集合名
//db_name 
//该方法是将所有的爬虫插入到爬虫表中，统一处理
let dbInfProc = (collectionShowName: string, collection_name: string): void => {
    let driver = new Driver();
    //let oneRecords = driver.findSync(dbinfoCollection, { name: collection_name, label: db_name });
    collectionShowName
    let oneRecords = driver.findSync(dbinfoCollection, { name: collection_name, collectionShowName: collectionShowName });
    if (oneRecords && oneRecords.length > 0) {
        // do nothing;
    } else {
        driver.insert(dbinfoCollection, [{ name: collection_name, collectionShowName: collectionShowName }]);
    }
};

// main
let start = function (): void {
    if (schdlObject.option == "exit") {
        return;
    }
    load_modules();
    var test_module = schdlObject.testmodule;
    if (schdlObject.option == "all") {
        for (var i = 0, j = modules.length; i < j; i++) {
            var m = modules[i];
            var agv = process.argv;
            console.log(agv[0], agv[1]);
            var out = fs.openSync('./out.log', 'a');
            var err = fs.openSync('./out.log', 'a');
            var child = child_process.spawn(agv[0], [agv[1], "-n", m.module_name], {
                detached: true,
                stdio: ['ignore', out, err]
            });
        }
    } else {
        if (schdlObject.option == "newitem") {
            for (let i = 0, j = modules.length; i < j; i++) {
                let m: Base = modules[i] as Base;
                let module_name = (<any>m).module_name;
                if ((<any>schdlObject).spidername == module_name) {
                    let spder_name = module_name;
                    if (test_module)
                        console.log("spider ", module_name, " in test mode");
                    else {
                        fs.writeFileSync(path.join(__dirname, spder_name), '');
                    }
                    //--dbinfo--
                    dbInfProc(m.collectionShowName, m.collection);//将每个爬虫插入到爬虫爬虫主表中
                    moduleHandle(m, test_module ? "test" : "");
                    console.log('module: ', m.name, ' is finished');
                    if (!test_module) {
                        fs.unlinkSync(path.join(__dirname, spder_name));
                    }
                }
            }
        }
    }
};

(<any>schdlObject).start = start;

export = schdlObject;
